package engine

import (
    "log"
    "spider/concurrent/fetcher"
    "spider/concurrent/parser"
)

type SimpleEngine struct {
}

func (SimpleEngine SimpleEngine) Run(seeds ...parser.Request)  {
    var requests []parser.Request
    
    requests = append(requests,seeds...)
    
    for len(requests) > 0 {
        // 相当于队列pop操作
        request := requests[0]
        requests = requests[1:]
    
        parseResult, err := SimpleEngine.worker(request)
        if err != nil {
            continue
        }
    
        // 将解析出来的request追加到request队列等待继续执行
        requests = append(requests,parseResult.Requests...)
    
        // 打印结果
        for _, value := range parseResult.ReponseSlice {
            log.Printf("got result %v",value)
        }
    }
}

// worker
// 将获取内容与解析内容封装到worker方法里边
// 方便我们使用协程开多个worker
func (SimpleEngine SimpleEngine) worker(request parser.Request) (parser.ParseResult,error) {
    
    //  获取内容
    log.Printf("Fetching %s",request.Url)
    content, err := fetcher.Fetch(request.Url)
    if err != nil {
        log.Printf("Fetcher: error fetching url %s : %v",request.Url,err)
        return parser.ParseResult{},nil
    }
    
    // 解析内容
    parseResult := request.Parser.Analysis(content)
    
    return parseResult,nil
}